import json
import time
import scrapy
import re
from wipo.libs import utils, config
from wipo.items import WipoBrandItem
import logging


class BrandSpider(scrapy.Spider):
    name = 'brand'
    allowed_domains = ['www3.wipo.int']
    start_urls = ['https://www3.wipo.int/branddb/en/']
    list_url = 'https://www3.wipo.int/branddb/jsp/select.jsp'
    ua = utils.get_ua()
    headers = {
        'User-Agent': ua,
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Referer': start_urls[0],
    }

    def parse(self, response):
        query_config = config.queryConfig
        qk_rg = re.compile(r'qk = "(.*)";')
        fo_rg = re.compile(r'src="../scripts/branddb-required.(\d+).min.js"')
        qk = re.findall(qk_rg, response.text)
        fo = re.findall(fo_rg, response.text)
        query_config['qi'] = '0-' + qk[1]
        query_config['_'] = fo[0]
        conn = config.conn
        keys = conn.hkeys('wipo-config')
        for k in keys:
            values = json.loads(conn.hget('wipo-config', k))
            status = values.get('status')
            start_from = values.get('startFrom')
            latest_date = values.get('latestDate')
            query_config.get('p').get('search').get('fq')[1]['te'] = k + 'TM'
            query_config.get('p').get('search').get('fq')[0]['te'] = status
            query_config.get('p')['start'] = int(start_from)
            qz = utils.encode_param(query_config)
            data = {
                'qz': qz
            }
            meta_dict = {
                'query_config': query_config,
                'latestDate': latest_date,
                'status': status
            }
            yield scrapy.FormRequest(self.list_url, callback=self.parse_list, method='POST', headers=self.headers,
                                     formdata=data, meta=meta_dict)

    def parse_list(self, response):
        meta = response.meta
        query_config = meta['query_config']
        text = json.loads(response.text)
        resp = text.get('response')
        num_found = resp.get('numFound')
        start = int(resp.get('start'))
        logging.info(num_found, start)
        # pages = numFound // 100
        # for i in range(1, pages+1):
        item = WipoBrandItem()
        item['resp'] = resp
        item['latestDate'] = response.meta['latestDate']
        item['status'] = response.meta['status']
        yield item
        if start + 100 < num_found:
            start += 100
            query_config.get('p')['start'] = start
            qz = utils.encode_param(query_config)
            data = {
                'qz': qz
            }
            meta_dict = {
                'query_config': query_config,
                'latestDate': response.meta['latestDate'],
                'status': response.meta['status']
            }
            time.sleep(1)
            yield scrapy.FormRequest(self.list_url, callback=self.parse_list, method='POST', headers=self.headers,
                                     formdata=data, meta=meta_dict)




